package com.inspur

import org.apache.spark.{SparkConf, SparkContext}

object PortraitUser { //计算买家画像

  def main(args: Array[String]): Unit = {
    val conf = new SparkConf()
      //      .setMaster("spark://192.168.66.88:7077")
      .setMaster("local")
      .setAppName("PortraitUser")
    val sc = new SparkContext(conf)

    val products = sc.textFile("hdfs://192.168.66.88:8020/0616/clear_data/products/part-00000")
      .map(_.split("\t"))
      .map(arr=>(arr(0), arr(1)))
    val order_goods = sc.textFile("hdfs://192.168.66.88:8020/0616/clear_data/order_goods/part-00000")
      .map(_.split("\t"))
      .map(arr=>(arr(1), arr(0)))

    products
      .join(order_goods)
      .map(x=>(x._2._2,x._2._1))
      .reduceByKey(_+_)
      .mapValues(value => {
        val values = value.split(";")
        var words = List[String]()
        for(word <- values) {
          if((!word.isEmpty) && (word.length != 0) && (!words.contains(word)))
            words = word::words
        }
        words.mkString(";")
      })
      .map(x=>x._1 + "\t" + x._2)
      .saveAsTextFile("hdfs://192.168.66.88:8020/0616/portrait_buyer")
  }

}
